package hasthi.tools.analyze;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.io.StringReader;
import java.text.DecimalFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import javax.xml.namespace.QName;

import org.apache.commons.math.stat.descriptive.DescriptiveStatistics;

import sun.nio.cs.ext.MacHebrew;

public class ParseHasthiData2 {
    private static List<Experiment> dataMap = new ArrayList<Experiment>();
    private static DecimalFormat df = new DecimalFormat(".0");
    
    private static String regex1 = "\\[([a-z]+).*?\\].*?faluire.*?Count=([0-9]+) DATA\\[(.*)\\]";
    private static Pattern pattern1 = Pattern.compile(regex1);
    
    private static String regex2 = ".*?Manager \\(.*?/([0-9]+)/([0-9]+)\\).*?\\(([0-9]+)MB\\)";
    private static Pattern pattern2 = Pattern.compile(regex2);
    private static String regex5 = ".*?Manager \\(.*?/(true|false)/([0-9]+)\\).*?\\(([0-9]+)MB\\)";
    private static Pattern pattern5 = Pattern.compile(regex5);
    
    private static String regex3 = ".*?Coordinator ([0-9]+).*?\\(([0-9]+)MB\\)";
    private static Pattern pattern3 = Pattern.compile(regex3);
    
    //private static String regex4 = "hasthi_([0-9]+)X([0-9]+)_.*?";
    private static String regex4 = "hasthi_[0-9]+X[0-9]+_([0-9.]+)_.*?";
    private static Pattern pattern4 = Pattern.compile(regex4);
    

    //WARN [Timer-1]Manager (31128398/true/484/0) Fri Mar 28 22:13:19 EDT 2008(21MB)
    //[aoc(ADAS1206756802257Fri_Mar_28_22_13_22_EDT_2008)]10/10 mean =43.60 median=43.27 cov =.06 size =10 min=40.62 max =48.16 faluireCount=0 DATA[41.369 40.622 41.469 43.527 43.015 40.852 45.037 45.768 46.171 48.165 ]
    //WARN [Timer-2]Coordinator 1446 R->11 Mgrs Fri Mar 28 22:11:13 EDT 2008 (34MB)
    //hasthi_5000X11_03_28_22_09
    //[moc(mng6abb4e65-b1fa-4740-bcea-a4795e0e860b)]10/10 mean =16181.12 median=14040.31 cov =.89 size =10 min=7.84 max =42663.20 faluireCeCount=0 DATA[11.766 37.293 48.642 9170.816 13177.926 14912.719 25556.413 41209.47 28782.387 27833.368 ]
    public static void main1(String[] args) throws Exception {
////        System.out.println(testRegex(pattern5, "WARN [Timer-1]Manager (17241377/false/0) Thu Mar 27 17:35:37 EDT 2008(40MB)"));
////        System.out.println(testRegex(pattern3, "WARN [Timer-2]Coordinator 1446 R->11 Mgrs Fri Mar 28 22:11:13 EDT 2008 (34MB)"));
//        //System.out.println(testRegex(pattern5, "WARN [Timer-1]Manager (30641031/true/0) Thu Mar 27 17:36:32 EDT 2008(45MB)"));
//        //System.out.println(testRegex(pattern1, "[waitc]10/10 mean =72897.83 median=72894.87 cov =.64 size =10 min=6.50 max =145788.11 faluireCount=0 DATA[6.497 25789.274 30026.768 55788.934 60002.07 85787.678 90002.633 115788.162 119998.132 145788.109 ]"));
            ParseHasthiData2 hasthiData = new ParseHasthiData2();
            hasthiData.parse(new File("/u/hperera/logs/hasthi_set1"));
//        //hasthiData.parse(new File("/u/hperera/logs/hasthi_tmp"));
//        //hasthiData.parse(new File("/u/hperera/logs/hasthi_set3"));
//        hasthiData.parse(new File("/u/hperera/logs/hasthi_sim2"));
    }
    
    
    public static void main(String[] args) throws Exception {
        ParseHasthiData2 hasthiData = new ParseHasthiData2();
        String[] dirs = null;
        if(args.length == 0){
            dirs = new String[]{"."};
        }else{
            dirs = args;
        }
        
        for(String dir:dirs){
            if(new File(dir,"hasthi.log").exists()){
                hasthiData.processAresultDir(new File(dir));
                hasthiData.printResults();
            }else{
                hasthiData.parse(new File(dir));
            }
        }
    }
    
    
    
    
    

    public static boolean testRegex(Pattern pattern,String str){
        return pattern.matcher(str).matches();
    }
      

    public void parse(File baseDir) throws Exception{
        File[] dirlist = baseDir.listFiles();
        for(File file:dirlist){
            processAresultDir(file);
        }
        printResults();
    }
    private void processAresultDir(File file) throws Exception {
        Matcher m = pattern4.matcher(file.getName());
        if(m.matches()){
//            int resourceCount = Integer.parseInt(m.group(1));
//            int managerCount = Integer.parseInt(m.group(2));
            String k1 = m.group(1);
//            String k2 = m.group(2);
//            String key = k2+"_"+k1;
            //System.out.println(file.getName() +" = " + key);
            Experiment experiment = parseADir(file);
            experiment.name = k1;
            dataMap.add(experiment);
            //break;
        }
    }
    
    public void printResults() throws Exception{
        Collections.sort(dataMap,new Comparator<Experiment>() {
            public int compare(Experiment o1, Experiment o2) {
                return o1.name.compareTo(o2.name);
            }
        
        });
        
        
        System.out.println("========================== Results ================================");
        //System.out.println("Key\t\tHB\tMLE2E\tMLR\tCLE2E\tCLR\tCAEx\tCAW\tMAEx\tMAW\tMM\tCM\tMgHB");
        System.out.println("#RCount\tCLE2E\tMngHB\tMM\tCM\tAHB\tMLE2E\tCLCi\tMHBCi\tAHBci\tMLCi\n");
        for(Experiment datastr:dataMap){
            System.out.println(datastr.name+"\t"+datastr.set1());
//            
//            String managerCount = keyParts[0];
//            String resourceCount = keyParts[1];
//            StringBuffer buffer = dataByManagerCount.get(managerCount);
//            if(buffer ==  null){
//                buffer = new StringBuffer();
//                dataByManagerCount.put(managerCount, buffer);
//            }
//            buffer.append(resourceCount).append("\t").append(datastr.set1()).append("\n");
        }
        System.out.println("===================================================================");
//        for(String key:keys){
//            System.out.println(dataMap.get(key).printdetails(key));
//        }
//        File f = new File("/u/hperera/logs/hasthi_data");
//        f.mkdirs();
//        for(String managerCount:dataByManagerCount.keySet()){
//            FileWriter out = new FileWriter(new File(f,"mng"+managerCount+".data"));
//            out.write("#RCount\tCLE2E\tMngHB\tMM\tCM\tAHB\tMLE2E\tCLCi\tMHBCi\tAHBci\tMLCi\n");
//            out.write(dataByManagerCount.get(managerCount).toString());
//            out.close();
//        }
//        
//        int[][] freqDatatCLE2E = new int[keys.size()][];
//        int[][] freqDatatMngHB = new int[keys.size()][];
//        int i = 0;
//        for(String key:keys){
//            Experiment exp = dataMap.get(key);
//            freqDatatCLE2E[i] = findFreqDistribution(exp.coordinatorLoopE2e);
//            freqDatatMngHB[i] = findFreqDistribution(exp.managerHeartbeat);
//            i++;
//        }    
//        writeFreqData(keys, freqDatatCLE2E, "/u/hperera/logs/hasthi_data/CLE2E_freq.data");
//        writeFreqData(keys, freqDatatMngHB, "/u/hperera/logs/hasthi_data/MngHB.data");
//        writePlotFile(keys, "/u/hperera/logs/hasthi_data/frq.plot");
        
    }
    
    public Experiment parseADir(File dir)throws Exception{
        Experiment experiment =  new Experiment();
        File[] files = dir.listFiles();
        for(int i = 0;i<files.length;i++){
            //if(files[i].getName().startsWith("action_corrdinator_")
            if(files[i].getName().startsWith("action_manager_")
                    ||files[i].getName().startsWith("hasthi_coordinator_")){
                parseDataFile(files[i], experiment,true);
            }else if(files[i].getName().startsWith("hasthi_manager_") 
                    || files[i].getName().startsWith("serviceodin")){
                parseDataFile(files[i], experiment,false);
            }else if(files[i].getName().startsWith("hasthi.log")){
                parseTraceFile(files[i], experiment);
            }
        }
        return experiment;
    }
    
    
    public void parseDataFile(File fileName, Experiment experiment,boolean coordinator) throws Exception{
        BufferedReader br = new BufferedReader(new FileReader(fileName));
        String line = br.readLine();
        try {
            while (line != null) {
                line = line.trim();
                Matcher matcher = pattern1.matcher(line);
                if (matcher.matches()) {
                    String name = matcher.group(1);
                    int faluireCount = Integer.parseInt(matcher.group(2));
                    String[] dataValues = matcher.group(3).split(" ");
                    DescriptiveStatistics data;
                    if(name.equals("aoc")){
                        data = experiment.agentHeartbeat;
                    }else if(name.equals("roc")){
                        data = coordinator?experiment.coordinatorLoopRules:experiment.managmentLoopRules;
                    }else if(name.equals("loc")){
                        data = coordinator?experiment.coordinatorLoopE2e:experiment.managmentLoopE2e;
                    }else if(name.equals("runc")){
                        data = coordinator?experiment.coordinatorActonsExec:experiment.managerActionsExec;
                    }else if(name.equals("waitc")){
                        data = coordinator?experiment.coordinatorActonsWait:experiment.managerActionsWait;
                    }else if(name.equals("moc")){
                        data = experiment.managerHeartbeat;
                    }else{
                        throw new Exception("Unknown name "+name);
                    }
                    for(String val:dataValues){
                        data.addValue(Double.parseDouble(val));
                    }
                    
                } else{
                    if(!line.startsWith("#")){
                        System.out.println("does not match "+line);    
                    }
                    
                }   
                line = br.readLine();
            }
           
        } catch (Throwable e) {
            //throw new Exception("Error at "+ line,e);
            System.out.println("Error at "+ line + e.getMessage());
        }finally{
            br.close();
        }
    }
    
    public void parseTraceFile(File fileName, Experiment experiment)throws Exception{
        BufferedReader br = new BufferedReader(new FileReader(fileName));
        String line = br.readLine();
        while (line != null) {
            line = line.trim();
            Matcher matcher1 = pattern2.matcher(line);
            Matcher matcher3 = pattern5.matcher(line);
            
            int  memeory = -1;
            int resourceCount = -1;
            if (matcher1.matches()) {
                resourceCount = Integer.parseInt(matcher1.group(1));
                if(resourceCount > 10){
                    memeory = Integer.parseInt(matcher1.group(3));
                }
            }else if(matcher3.matches()){
                resourceCount = Integer.parseInt(matcher3.group(2));
                if(resourceCount > 10){
                    memeory = Integer.parseInt(matcher3.group(3));
                    
                }
            }
            if(memeory > 0){
                experiment.memoryManager.addValue(memeory);
            }else{
                Matcher matcher2 = pattern3.matcher(line);
                if(matcher2.matches()){
                    resourceCount = Integer.parseInt(matcher2.group(1));
                    memeory = Integer.parseInt(matcher2.group(2));
                    experiment.memeoryCoordinator.addValue(memeory);
                }else{
                    if(resourceCount == -1 && !line.startsWith("#") && (line.contains("Manager ")||line.contains("Coordinator "))){
                        System.out.println("Can not handle "+line);
                    }
                }
            }
           
            line = br.readLine();
        }
        br.close();
    }
    

    public static class Experiment {
        String name;
        DescriptiveStatistics agentHeartbeat = DescriptiveStatistics.newInstance();
        DescriptiveStatistics managmentLoopE2e= DescriptiveStatistics.newInstance();
        DescriptiveStatistics managmentLoopRules= DescriptiveStatistics.newInstance();
        DescriptiveStatistics coordinatorLoopE2e= DescriptiveStatistics.newInstance();
        DescriptiveStatistics coordinatorLoopRules= DescriptiveStatistics.newInstance();
        DescriptiveStatistics coordinatorActonsExec= DescriptiveStatistics.newInstance();
        DescriptiveStatistics coordinatorActonsWait= DescriptiveStatistics.newInstance();
        DescriptiveStatistics managerActionsExec= DescriptiveStatistics.newInstance();
        DescriptiveStatistics managerActionsWait= DescriptiveStatistics.newInstance();
        DescriptiveStatistics memoryManager= DescriptiveStatistics.newInstance();
        DescriptiveStatistics memeoryCoordinator= DescriptiveStatistics.newInstance();
        DescriptiveStatistics managerHeartbeat= DescriptiveStatistics.newInstance();
        
        public String printdetails(String name){
            StringBuffer buf = new StringBuffer();
            buf.append("### ").append(name).append(" ####\n");
            buf.append("Name\t\t\tmean\tmedian\tcov\tsize\tmin\tmax\n");
            
            buf.append("agentHeartbeat\t").append(formatOne(agentHeartbeat));
            buf.append("managmentLoopE2e\t").append(formatOne(managmentLoopE2e));
            buf.append("managmentLoopRules\t").append(formatOne(managmentLoopRules));
            buf.append("coordinatorLoopE2e\t").append(formatOne(coordinatorLoopE2e));
            buf.append("coordinatorLoopRules\t").append(formatOne(coordinatorLoopRules));
            buf.append("coordinatorActonsExec\t").append(formatOne(coordinatorActonsExec));
            buf.append("coordinatorActonsWait\t").append(formatOne(coordinatorActonsWait));
            buf.append("managerActionsExec\t").append(formatOne(managerActionsExec));
            buf.append("managerActionsWait\t").append(formatOne(managerActionsWait));
            buf.append("memoryManager\t\t").append(formatOne(memoryManager));
            buf.append("memeoryCoordinator\t").append(formatOne(memeoryCoordinator));
            buf.append("managerHeartbeat\t").append(formatOne(managerHeartbeat));
            return buf.toString();
        }
        
        
        public String formatOne(DescriptiveStatistics data){
            double mean = data.getMean();
            double max = data.getMax();
            double min = data.getMin();
            double size = data.getValues().length;
            double median = data.getPercentile(50);
            double cov = data.getStandardDeviation()/mean;
       
            StringBuffer buf = new StringBuffer();
            buf.append(df.format(mean)).append("\t").append(df.format(median)).append("\t")
            .append(df.format(cov)).append("\t").append(df.format(size)).append("\t").append(df.format(min)).append("\t").append(df.format(max));
            buf.append("\n");
            return buf.toString();
        }
        
        public String toString(){
            StringBuffer buf = new StringBuffer();
            buf.append(df.format(agentHeartbeat.getMean())).append("\t");
            buf.append(df.format(managmentLoopE2e.getMean())).append("\t");
            buf.append(df.format(managmentLoopRules.getMean())).append("\t");
            buf.append(df.format(coordinatorLoopE2e.getMean())).append("\t");
            buf.append(df.format(coordinatorLoopRules.getMean())).append("\t");
            buf.append(df.format(coordinatorActonsExec.getMean())).append("\t");
            buf.append(df.format(coordinatorActonsWait.getMean())).append("\t");
            buf.append(df.format(managerActionsExec.getMean())).append("\t");
            buf.append(df.format(managerActionsWait.getMean())).append("\t");
            buf.append(df.format(memoryManager.getMean())).append("\t");
            buf.append(df.format(memeoryCoordinator.getMean())).append("\t");
            buf.append(df.format(managerHeartbeat.getMean())).append("\t");
            
            buf.append(df.format(agentHeartbeat.getPercentile(5))).append("\t");
            buf.append(df.format(agentHeartbeat.getPercentile(90))).append("\t");
            buf.append(df.format(managmentLoopE2e.getPercentile(5))).append("\t");
            buf.append(df.format(managmentLoopE2e.getPercentile(95))).append("\t");
            buf.append(df.format(managmentLoopRules.getPercentile(5))).append("\t");
            buf.append(df.format(managmentLoopRules.getPercentile(95))).append("\t");
            
            buf.append(df.format(coordinatorLoopE2e.getPercentile(5))).append("\t");
            buf.append(df.format(coordinatorLoopE2e.getPercentile(95))).append("\t");
            buf.append(df.format(coordinatorLoopRules.getPercentile(5))).append("\t");
            buf.append(df.format(coordinatorLoopRules.getPercentile(95))).append("\t");
            
            buf.append(df.format(coordinatorActonsExec.getPercentile(5))).append("\t");
            buf.append(df.format(coordinatorActonsExec.getPercentile(95))).append("\t");
            
            buf.append(df.format(coordinatorActonsWait.getPercentile(5))).append("\t");
            buf.append(df.format(coordinatorActonsWait.getPercentile(95))).append("\t");
            
            buf.append(df.format(managerActionsExec.getPercentile(5))).append("\t");
            buf.append(df.format(managerActionsExec.getPercentile(95))).append("\t");
            
            buf.append(df.format(managerActionsWait.getPercentile(5))).append("\t");
            buf.append(df.format(managerActionsWait.getPercentile(95))).append("\t");
            
            buf.append(df.format(memoryManager.getPercentile(5))).append("\t");
            buf.append(df.format(memoryManager.getPercentile(95))).append("\t");
            
            buf.append(df.format(memeoryCoordinator.getPercentile(5))).append("\t");
            buf.append(df.format(memeoryCoordinator.getPercentile(95))).append("\t");
            
            buf.append(df.format(managerHeartbeat.getPercentile(5))).append("\t");
            buf.append(df.format(managerHeartbeat.getPercentile(95))).append("\t");

            return buf.toString();
        }
        
        public double confidence95Interval(DescriptiveStatistics data){
            double size = data.getValues().length;
            return 1.645*data.getStandardDeviation()/Math.sqrt(size);
        }
        
        
        public String set1(){
            StringBuffer buf = new StringBuffer();
            buf.append(df.format(coordinatorLoopE2e.getMean())).append("\t");
            buf.append(df.format(managerHeartbeat.getMean())).append("\t");
            buf.append(df.format(memoryManager.getMax())).append("\t");
            buf.append(df.format(memeoryCoordinator.getMax())).append("\t");
            buf.append(df.format(agentHeartbeat.getMean())).append("\t");
            buf.append(df.format(managmentLoopE2e.getMean())).append("\t");
            
            
            buf.append(df.format(confidence95Interval(coordinatorLoopE2e))).append("\t");
            buf.append(df.format(confidence95Interval(managerHeartbeat))).append("\t");
            buf.append(df.format(confidence95Interval(agentHeartbeat))).append("\t");
            buf.append(df.format(confidence95Interval(managmentLoopE2e))).append("\t");
            
            return buf.toString();
        }

        
    }
    
    public void writePlotFile(Set<String> keys,String file) throws IOException{
        FileWriter writer = new FileWriter(file);
        writer.write("set terminal png\n");
        writer.write("set output \"frqCLE2E.png\"\n");
        writer.write("set title \"Frequnecy vs. Steps\"\n");;
        writer.write("set ylabel \"Frequnecy\"\n");
        writer.write("set xlabel \"Step\"\n");
        writer.write("set key right top\n");
        writer.write("plot "); 
        int i = 0;
        for(String key:keys){
            writer.write("\t\"CLE2E_freq.data\" using 1:"+(i+2)+" title \""+key+"\" with linespoints");
            if(i != keys.size()-1){
                writer.write(", \\");         
            }
            writer.write("\n");
            i++;
        }
        
        writer.write("set output \"frqMngHB.png\"\n");
        writer.write("plot ");
        i = 0;
        for(String key:keys){
            writer.write("\t\"MngHB.data\" using 1:"+(i+2)+" title \""+key+"\" with linespoints");
            if(i != keys.size()-1){
                writer.write(", \\");         
            }
            writer.write("\n");
            i++;
        }
        writer.close();
        
    }
    
    
    public void writeFreqData(Set<String> keys,int[][] data,String file) throws IOException{
        FileWriter writer = new FileWriter(file);
        writer.write("#");
        for(String name:keys){
            writer.write(format(name));
        }
        writer.write("\n");
        int expCount = data.length;
        int freDataCount = data[0].length;
        for(int i=0;i<freDataCount;i++){
            writer.write(format(String.valueOf(i)));
            for(int j=0;j<expCount;j++){
                writer.write(format(String.valueOf(data[j][i])));
            }
            writer.write("\n");
        }
        writer.close();
        
    }
    public String format(String str){
        if(str.length() <12){
            StringBuffer buf = new StringBuffer(str);
            int padSize = 12 -str.length();
            for(int i = 0;i<padSize;i++){
                buf.append(" ");
            }
            return buf.toString();
        }
        return str;
    }
    
    public int[] findFreqDistribution(DescriptiveStatistics stats){
        int stepCount = 100;
        double step = (stats.getMax() - stats.getMin())/stepCount;
        int[] freq = new int[stepCount];
        
        double[] data = stats.getValues();
        for(double val:data){
            int index = (int)Math.floor((val -stats.getMin())/step);
            if(index == 100){
                index--;
            }
            freq[index]++;
        }
        return freq;
    }
    

}